import fitz  # PyMuPDF
import re
import os
import csv
from typing import List, Dict, Tuple, Optional, Any
from collections import defaultdict

class EnhancedTableDetector:
    def __init__(self, pdf_path: str):
        self.doc = fitz.open(pdf_path)
        self.pages = [self.doc.load_page(i) for i in range(len(self.doc))]
        self.pdf_path = pdf_path
        print(f"已加载PDF文件: {pdf_path}")
        print(f"共 {len(self.pages)} 页")

    def extract_page_elements(self, page_num: int) -> Dict[str, Any]:
        """
        提取页面中的所有元素：文本块、线条和区域信息
        增强版：添加更多的文本特征提取
        """
        page = self.pages[page_num]
        elements = {
            "text_blocks": [],
            "lines": [],
            "tables": [],
            "page_rect": page.rect,
            "words": []
        }

        # 提取文本块
        text_dict = page.get_text("dict")
        if "blocks" in text_dict:
            for block in text_dict["blocks"]:
                if block["type"] == 0:  # 文本块
                    text_lines = []
                    words = []
                    for line in block.get("lines", []):
                        line_text = " ".join([span["text"] for span in line.get("spans", [])])
                        text_lines.append(line_text)

                        # 提取单个单词信息
                        for span in line.get("spans", []):
                            if "text" in span and "bbox" in span:
                                words.append({
                                    "text": span["text"],
                                    "bbox": span["bbox"],
                                    "font": span.get("font", "unknown"),
                                    "size": span.get("size", 0)
                                })

                    elements["text_blocks"].append({
                        "bbox": block["bbox"],
                        "text": "\n".join(text_lines),
                        "full_block": block,
                        "words": words
                    })
                    elements["words"].extend(words)

        # 提取线条和表格框线
        drawings = page.get_drawings()
        for drawing in drawings:
            if "items" in drawing:
                for item in drawing["items"]:
                    if item[0] == "l":  # 线条
                        if len(item) >= 2:
                            line_points = item[1]
                            if isinstance(line_points, (list, tuple)) and len(line_points) == 4:
                                x0, y0, x1, y1 = line_points
                                elements["lines"].append({
                                    "type": "line",
                                    "points": [(x0, y0), (x1, y1)],
                                    "bbox": (min(x0, x1), min(y0, y1), max(x0, x1), max(y0, y1))
                                })
                    elif item[0] == "re":  # 矩形
                        if len(item) >= 2:
                            rect_params = item[1]
                            if isinstance(rect_params, (list, tuple)) and len(rect_params) == 4:
                                x, y, w, h = rect_params
                                elements["lines"].append({
                                    "type": "rect",
                                    "bbox": (x, y, x + w, y + h)
                                })

        return elements

    def detect_tables(self, page_num: int, use_lines: bool = True, use_text: bool = True) -> List[Dict]:
        """
        综合检测页面中的表格，结合多种方法
        """
        elements = self.extract_page_elements(page_num)
        tables = []

        # 方法1: 基于线条的表格检测
        if use_lines:
            line_tables = self._detect_tables_from_lines(elements["lines"])
            tables.extend(line_tables)

        # 方法2: 基于文本布局的表格检测（适合无框线表格）
        if use_text:
            text_tables = self._detect_tables_from_text_layout(elements["text_blocks"], elements["words"])
            tables.extend(text_tables)

        # 去重相似的表格检测结果
        tables = self._deduplicate_tables(tables)

        return tables

    def _detect_tables_from_lines(self, lines: List[Dict]) -> List[Dict]:
        """
        从线条中检测表格
        改进版：更宽松的线条检测阈值
        """
        tables = []

        # 提取水平线和垂直线，使用更宽松的阈值
        horizontal_lines = []
        vertical_lines = []

        for line in lines:
            bbox = line["bbox"]
            width = bbox[2] - bbox[0]
            height = bbox[3] - bbox[1]

            # 水平线判断：高度很小
            if height < 3 and width > 20:  # 更宽松的判断
                horizontal_lines.append({
                    "y": (bbox[1] + bbox[3]) / 2,
                    "x0": bbox[0],
                    "x1": bbox[2],
                    "bbox": bbox,
                    "length": width
                })
            # 垂直线判断：宽度很小
            elif width < 3 and height > 20:  # 更宽松的判断
                vertical_lines.append({
                    "x": (bbox[0] + bbox[2]) / 2,
                    "y0": bbox[1],
                    "y1": bbox[3],
                    "bbox": bbox,
                    "length": height
                })

        # 按Y坐标排序水平线
        horizontal_lines.sort(key=lambda x: x["y"])
        # 按X坐标排序垂直线
        vertical_lines.sort(key=lambda x: x["x"])

        # 如果有足够的线条，可能构成表格
        if len(horizontal_lines) >= 2 and len(vertical_lines) >= 2:
            # 查找表格边界
            min_x = min(vl["x"] for vl in vertical_lines)
            max_x = max(vl["x"] for vl in vertical_lines)
            min_y = min(hl["y"] for hl in horizontal_lines)
            max_y = max(hl["y"] for hl in horizontal_lines)

            # 计算线条密度，提高检测准确性
            width = max_x - min_x
            height = max_y - min_y

            if width > 50 and height > 50:  # 确保表格足够大
                tables.append({
                    "bbox": (min_x, min_y, max_x, max_y),
                    "horizontal_lines": horizontal_lines,
                    "vertical_lines": vertical_lines,
                    "confidence": min(1.0, (len(horizontal_lines) + len(vertical_lines)) / 20),
                    "type": "line_based"
                })

        return tables

    def _detect_tables_from_text_layout(self, text_blocks: List[Dict], words: List[Dict]) -> List[Dict]:
        """
        从文本布局检测表格（无框线表格检测）
        增强版：基于文本对齐和列结构的检测
        """
        if not words or len(words) < 10:  # 确保有足够的文本
            return []

        tables = []

        # 按y坐标排序所有单词
        words.sort(key=lambda w: w["bbox"][1])

        # 基于单词的垂直位置分组，找到可能的行
        lines = []
        current_line = [words[0]]
        current_y = words[0]["bbox"][1]

        for word in words[1:]:
            y = word["bbox"][1]
            # 如果y坐标接近当前行，认为是同一行
            if abs(y - current_y) < 3:  # 更宽松的行间距判断
                current_line.append(word)
            else:
                lines.append(current_line)
                current_line = [word]
                current_y = y
        lines.append(current_line)

        # 按x坐标排序每行的单词
        for i in range(len(lines)):
            lines[i].sort(key=lambda w: w["bbox"][0])

        # 分析列结构
        if len(lines) >= 3:  # 至少需要3行来检测表格结构
            # 计算每行单词的x坐标分布
            column_positions = []

            for line in lines:
                if len(line) >= 2:  # 每行至少有2个单词
                    x_positions = [word["bbox"][0] for word in line]
                    column_positions.append(x_positions)

            # 寻找共同的列位置（表格特征）
            if len(column_positions) >= 3:
                # 统计x坐标频率，找出可能的列分隔位置
                x_freq = defaultdict(int)
                for positions in column_positions:
                    for x in positions:
                        # 对x坐标进行分组（允许一定误差）
                        rounded_x = round(x / 5) * 5  # 5像素误差
                        x_freq[rounded_x] += 1

                # 找出高频x坐标作为列位置
                common_x = [x for x, count in x_freq.items() if count >= len(column_positions) * 0.6]
                common_x.sort()

                # 如果有至少2个列位置，可能是表格
                if len(common_x) >= 2:
                    # 确定表格边界
                    min_y = min(line[0]["bbox"][1] for line in lines[:5])  # 前5行的最小y
                    max_y = max(line[-1]["bbox"][3] for line in lines[-5:])  # 后5行的最大y
                    min_x = min(common_x)
                    max_x = max(common_x)

                    tables.append({
                        "bbox": (min_x, min_y, max_x, max_y),
                        "confidence": min(1.0, len(common_x) / 10),
                        "type": "text_based",
                        "column_positions": common_x
                    })

        return tables

    def _deduplicate_tables(self, tables: List[Dict]) -> List[Dict]:
        """
        去重相似的表格检测结果
        """
        if len(tables) <= 1:
            return tables

        unique_tables = []
        for table in tables:
            is_duplicate = False
            for unique_table in unique_tables:
                if self._tables_are_similar(table["bbox"], unique_table["bbox"]):
                    is_duplicate = True
                    break
            if not is_duplicate:
                unique_tables.append(table)

        return unique_tables

    def _tables_are_similar(self, rect1, rect2, tolerance=10.0) -> bool:
        """
        检查两个表格区域是否相似
        """
        # 检查边界是否接近
        return (abs(rect1[0] - rect2[0]) < tolerance and
                abs(rect1[1] - rect2[1]) < tolerance and
                abs(rect1[2] - rect2[2]) < tolerance and
                abs(rect1[3] - rect2[3]) < tolerance)

    def detect_cross_page_tables(self, margin_threshold: float = 30.0) -> List[Tuple[int, int, Dict, Dict]]:
        """
        增强版跨页表格检测，使用多种策略
        返回格式: [(page1, page2, table1, table2), ...]
        """
        cross_page_tables = []

        for i in range(len(self.pages) - 1):
            print(f"\n检查第 {i+1} 页和第 {i+2} 页之间的跨页表格...")

            # 获取两页的表格
            page1_tables = self.detect_tables(i)
            page2_tables = self.detect_tables(i + 1)

            # 获取页面边界
            page1_rect = self.pages[i].rect
            page2_rect = self.pages[i + 1].rect

            print(f"  第 {i+1} 页检测到 {len(page1_tables)} 个表格")
            print(f"  第 {i+2} 页检测到 {len(page2_tables)} 个表格")

            # 方法1: 检查页面1底部和页面2顶部的表格
            page1_bottom_tables = []
            for table in page1_tables:
                table_bbox = table["bbox"]
                # 检查表格是否靠近页面底部
                if page1_rect.y1 - table_bbox[3] <= margin_threshold * 2:  # 更大的区域
                    page1_bottom_tables.append(table)
                    print(f"  第 {i+1} 页底部表格: {table_bbox}")

            page2_top_tables = []
            for table in page2_tables:
                table_bbox = table["bbox"]
                # 检查表格是否靠近页面顶部
                if table_bbox[1] - page2_rect.y0 <= margin_threshold * 2:  # 更大的区域
                    page2_top_tables.append(table)
                    print(f"  第 {i+2} 页顶部表格: {table_bbox}")

            # 检查表格是否对齐
            for table1 in page1_bottom_tables:
                for table2 in page2_top_tables:
                    if self._are_tables_aligned(table1["bbox"], table2["bbox"]):
                        cross_page_tables.append((i, i + 1, table1, table2))
                        print(f"  [+] 检测到跨页表格: 基于表格边界对齐")
                        break
                if any(i == pair[0] and i+1 == pair[1] for pair in cross_page_tables):
                    break

            # 如果已经检测到表格，跳过其他检测方法
            if any(i == pair[0] and i+1 == pair[1] for pair in cross_page_tables):
                continue

            # 方法2: 基于文本连续性的跨页表格检测（无需预先检测表格）
            if self._check_cross_page_text_structure(i, i+1, margin_threshold):
                cross_page_tables.append((i, i+1, None, None))
                print(f"  [+] 检测到跨页表格: 基于文本结构连续性")

        return cross_page_tables

    def _are_tables_aligned(self, rect1, rect2, tolerance=15.0) -> bool:
        """
        检查两个表格是否对齐，使用更宽松的阈值
        """
        # 检查左右边界是否对齐
        left_aligned = abs(rect1[0] - rect2[0]) < tolerance
        right_aligned = abs(rect1[2] - rect2[2]) < tolerance

        # 检查宽度是否相似
        width1 = rect1[2] - rect1[0]
        width2 = rect2[2] - rect2[0]
        width_similar = abs(width1 - width2) < max(width1, width2) * 0.2  # 宽度差异小于20%

        return (left_aligned and right_aligned) or width_similar

    def _check_cross_page_text_structure(self, page1_num: int, page2_num: int, margin_threshold: float) -> bool:
        """
        直接检查两页之间的文本结构连续性，不依赖预先检测的表格
        """
        # 获取页面元素
        page1_elements = self.extract_page_elements(page1_num)
        page2_elements = self.extract_page_elements(page2_num)

        # 获取页面边界
        page1_rect = page1_elements["page_rect"]
        page2_rect = page2_elements["page_rect"]

        # 获取页面1底部和页面2顶部的文本块
        bottom_texts = []
        for block in page1_elements["text_blocks"]:
            if block["bbox"][3] > page1_rect.y1 - margin_threshold * 3:  # 更大的底部区域
                bottom_texts.append(block)

        top_texts = []
        for block in page2_elements["text_blocks"]:
            if block["bbox"][1] < page2_rect.y0 + margin_threshold * 3:  # 更大的顶部区域
                top_texts.append(block)

        if not bottom_texts or not top_texts:
            return False

        # 提取文本内容
        bottom_text = " ".join([block["text"] for block in bottom_texts])
        top_text = " ".join([block["text"] for block in top_texts])

        # 增强的文本连续性检查
        # 检查1: 数字序列连续性
        bottom_numbers = re.findall(r'\d+(?:\.\d+)?', bottom_text)  # 支持小数
        top_numbers = re.findall(r'\d+(?:\.\d+)?', top_text)

        if bottom_numbers and top_numbers:
            try:
                # 检查最后一个底部数字和第一个顶部数字是否连续或形成序列
                last_bottom = float(bottom_numbers[-1])
                first_top = float(top_numbers[0])

                # 检查多种序列模式：+1, 递增, 相同格式等
                if (abs(first_top - last_bottom - 1) < 0.1 or  # 连续数字
                    abs(first_top - last_bottom) < 10 and first_top > last_bottom):  # 递增序列
                    return True
            except:
                pass

        # 检查2: 文本列结构相似性
        # 分析底部和顶部文本块的列分布
        bottom_columns = self._analyze_text_columns(bottom_texts)
        top_columns = self._analyze_text_columns(top_texts)

        # 如果列数相同且位置相似，很可能是同一表格
        if (len(bottom_columns) > 0 and len(top_columns) > 0 and
            abs(len(bottom_columns) - len(top_columns)) <= 1):
            # 计算列位置相似度
            min_len = min(len(bottom_columns), len(top_columns))
            position_matches = 0

            for i in range(min_len):
                if abs(bottom_columns[i] - top_columns[i]) < 20:  # 更宽松的列位置匹配
                    position_matches += 1

            if position_matches / min_len > 0.6:  # 60%以上的列位置匹配
                return True

        # 检查3: 文本行结构相似性
        # 分析行长度分布
        bottom_line_lengths = []
        for block in bottom_texts:
            bottom_line_lengths.extend([len(line.strip()) for line in block["text"].split('\n') if line.strip()])

        top_line_lengths = []
        for block in top_texts:
            top_line_lengths.extend([len(line.strip()) for line in block["text"].split('\n') if line.strip()])

        if len(bottom_line_lengths) > 2 and len(top_line_lengths) > 2:
            # 计算行长度分布的相似度
            bottom_avg = sum(bottom_line_lengths) / len(bottom_line_lengths)
            top_avg = sum(top_line_lengths) / len(top_line_lengths)

            # 如果平均长度相近
            if abs(bottom_avg - top_avg) < max(bottom_avg, top_avg, 1) * 0.4:  # 40%的容差
                # 检查长度分布的形状
                bottom_std = self._calculate_std(bottom_line_lengths)
                top_std = self._calculate_std(top_line_lengths)

                if abs(bottom_std - top_std) < max(bottom_std, top_std, 1) * 0.5:  # 50%的容差
                    return True

        return False

    def _analyze_text_columns(self, text_blocks: List[Dict]) -> List[float]:
        """
        分析文本块的列分布
        """
        if not text_blocks:
            return []

        # 收集所有文本块的左边界
        left_edges = []
        for block in text_blocks:
            left_edges.append(block["bbox"][0])

        # 对左边界进行聚类，找出可能的列位置
        if len(left_edges) > 1:
            left_edges.sort()
            clusters = []
            current_cluster = [left_edges[0]]

            for edge in left_edges[1:]:
                # 如果当前边界与聚类中心的距离小于阈值，则加入同一聚类
                cluster_center = sum(current_cluster) / len(current_cluster)
                if abs(edge - cluster_center) < 15:  # 15像素的聚类阈值
                    current_cluster.append(edge)
                else:
                    # 完成当前聚类，开始新聚类
                    clusters.append(cluster_center)
                    current_cluster = [edge]

            # 添加最后一个聚类
            if current_cluster:
                clusters.append(sum(current_cluster) / len(current_cluster))

            # 排序聚类中心
            clusters.sort()
            return clusters

        return left_edges

    def _calculate_std(self, values: List[float]) -> float:
        """
        计算标准差
        """
        if len(values) <= 1:
            return 0

        mean = sum(values) / len(values)
        variance = sum((x - mean) ** 2 for x in values) / len(values)
        return variance ** 0.5

    def extract_table_data(self, page_num: int, region: Optional[Tuple[float, float, float, float]] = None) -> List[List[str]]:
        """
        提取指定区域内的表格数据
        增强版：优化文本块分组和行列组织
        """
        elements = self.extract_page_elements(page_num)
        text_blocks = elements["text_blocks"]

        if region:
            # 只提取指定区域内的文本块
            x0, y0, x1, y1 = region
            text_blocks = [block for block in text_blocks
                          if (block["bbox"][0] <= x1 and  # 块的左边界在区域右边界左侧
                              block["bbox"][2] >= x0 and  # 块的右边界在区域左边界右侧
                              block["bbox"][1] <= y1 and  # 块的上边界在区域下边界上方
                              block["bbox"][3] >= y0)]    # 块的下边界在区域上边界下方

        if not text_blocks:
            return []

        # 使用单词级别信息进行更精确的表格提取
        all_words = []
        for block in text_blocks:
            all_words.extend(block["words"])

        if not all_words:
            # 如果没有单词级别信息，回退到文本块级别
            return self._extract_table_from_blocks(text_blocks)

        # 按Y坐标排序单词，组织成行
        all_words.sort(key=lambda w: w["bbox"][1])

        rows = []
        current_row = [all_words[0]]
        current_y = all_words[0]["bbox"][1]

        # 计算页面平均行高，用于动态调整行分组阈值
        line_heights = []
        for i in range(1, min(20, len(all_words))):  # 采样前20个单词计算行高
            if abs(all_words[i]["bbox"][1] - all_words[i-1]["bbox"][1]) > 1:
                line_heights.append(abs(all_words[i]["bbox"][1] - all_words[i-1]["bbox"][1]))

        # 设置行分组阈值
        line_height_threshold = 12  # 默认阈值
        if line_heights:
            avg_line_height = sum(line_heights) / len(line_heights)
            line_height_threshold = avg_line_height * 0.8  # 使用平均行高的80%作为阈值

        # 组织行
        for word in all_words[1:]:
            y = word["bbox"][1]
            # 如果y坐标接近当前行，认为是同一行
            if abs(y - current_y) < line_height_threshold:
                current_row.append(word)
            else:
                # 完成当前行，按X坐标排序
                current_row.sort(key=lambda w: w["bbox"][0])
                rows.append([w["text"] for w in current_row])
                # 开始新行
                current_row = [word]
                current_y = y

        # 添加最后一行
        if current_row:
            current_row.sort(key=lambda w: w["bbox"][0])
            rows.append([w["text"] for w in current_row])

        return rows

    def _extract_table_from_blocks(self, text_blocks: List[Dict]) -> List[List[str]]:
        """
        从文本块级别提取表格数据（备用方法）
        """
        # 按Y坐标排序文本块
        text_blocks.sort(key=lambda x: x["bbox"][1])

        # 合并接近的行
        rows = []
        current_row = []
        last_y = None

        for block in text_blocks:
            bbox = block["bbox"]
            y_center = (bbox[1] + bbox[3]) / 2

            # 确定是否为新行的阈值
            if last_y is None:
                current_row.append(block)
                last_y = y_center
            else:
                # 计算与上一行的垂直距离
                vertical_distance = abs(y_center - last_y)

                # 如果距离小于阈值，认为是同一行
                if vertical_distance < 15:  # 更宽松的阈值
                    current_row.append(block)
                else:
                    # 完成当前行
                    if current_row:
                        # 按X坐标排序当前行的文本块
                        current_row.sort(key=lambda x: x["bbox"][0])
                        rows.append([b["text"] for b in current_row])
                    # 开始新行
                    current_row = [block]
                    last_y = y_center

        # 添加最后一行
        if current_row:
            current_row.sort(key=lambda x: x["bbox"][0])
            rows.append([b["text"] for b in current_row])

        return rows

    def merge_cross_page_tables(self, margin_threshold: float = 30.0):
        """
        合并检测到的跨页表格
        增强版：更灵活的区域提取和数据合并
        """
        cross_page_pairs = self.detect_cross_page_tables(margin_threshold)
        merged_tables = {}

        for page1, page2, table1, table2 in cross_page_pairs:
            print(f"\n处理跨页表格: 第 {page1+1} 页和第 {page2+1} 页")

            # 获取页面边界
            page1_rect = self.pages[page1].rect
            page2_rect = self.pages[page2].rect

            # 确定提取区域
            # 对于第一页，提取底部区域
            page1_region = (page1_rect.x0, page1_rect.y1 - margin_threshold * 4,  # 更大的提取区域
                           page1_rect.x1, page1_rect.y1)

            # 对于第二页，提取顶部区域
            page2_region = (page2_rect.x0, page2_rect.y0,
                           page2_rect.x1, page2_rect.y0 + margin_threshold * 4)  # 更大的提取区域

            # 如果有表格边界信息，使用更精确的提取区域
            if table1 and "bbox" in table1:
                page1_region = (max(page1_region[0], table1["bbox"][0] - 10),
                               max(page1_region[1], table1["bbox"][1] - 10),
                               min(page1_region[2], table1["bbox"][2] + 10),
                               min(page1_region[3], table1["bbox"][3] + 10))

            if table2 and "bbox" in table2:
                page2_region = (max(page2_region[0], table2["bbox"][0] - 10),
                               max(page2_region[1], table2["bbox"][1] - 10),
                               min(page2_region[2], table2["bbox"][2] + 10),
                               min(page2_region[3], table2["bbox"][3] + 10))

            print(f"  提取区域 - 第 {page1+1} 页: {page1_region}")
            print(f"  提取区域 - 第 {page2+1} 页: {page2_region}")

            # 提取表格数据
            table_part1 = self.extract_table_data(page1, page1_region)
            table_part2 = self.extract_table_data(page2, page2_region)

            print(f"  从第 {page1+1} 页提取了 {len(table_part1)} 行数据")
            print(f"  从第 {page2+1} 页提取了 {len(table_part2)} 行数据")

            # 去重处理：如果两部分有重叠的行，只保留一行
            if table_part1 and table_part2:
                # 检查最后几行和第一几行是否相似
                last_rows_page1 = table_part1[-3:] if len(table_part1) >= 3 else table_part1
                first_rows_page2 = table_part2[:3] if len(table_part2) >= 3 else table_part2

                # 更智能的去重逻辑
                to_remove = self._find_overlapping_rows(last_rows_page1, first_rows_page2)

                if to_remove > 0:
                    print(f"  检测到 {to_remove} 行重叠内容，已去除重复")
                    table_part1 = table_part1[:-to_remove]

            # 合并表格
            merged_table = table_part1 + table_part2

            merged_tables[(page1, page2)] = merged_table
            print(f"  合并后的表格有 {len(merged_table)} 行")

            # 显示前几行和后几行作为示例
            print("  前几行数据:")
            for i, row in enumerate(merged_table[:9]):
                print(f"    行 {i+1}: {row}")

            if len(merged_table) > 6:
                print("    ...")
                print("  后几行数据:")
                for i, row in enumerate(merged_table[-3:], len(merged_table)-2):
                    print(f"    行 {i}: {row}")
            elif len(merged_table) > 3:
                print("    ...")
                for i, row in enumerate(merged_table[3:], 4):
                    print(f"    行 {i}: {row}")

        return merged_tables

    def _find_overlapping_rows(self, rows1: List[List[str]], rows2: List[List[str]]) -> int:
        """
        查找两行集合之间的重叠行数
        增强版：更智能的重复检测
        """
        # 反向遍历第一组行，正向遍历第二组行
        max_overlap = 0

        for i in range(len(rows1)):
            for j in range(len(rows2)):
                if self._rows_are_similar(rows1[len(rows1)-1-i], rows2[j]):
                    # 找到相似行后，检查后续行是否也相似
                    current_overlap = 1
                    k = 1
                    while (len(rows1)-1-i-k >= 0 and j+k < len(rows2) and
                           self._rows_are_similar(rows1[len(rows1)-1-i-k], rows2[j+k])):
                        current_overlap += 1
                        k += 1

                    max_overlap = max(max_overlap, current_overlap)
                    break

            if max_overlap > 0:
                break

        return max_overlap

    def _rows_are_similar(self, row1: List[str], row2: List[str]) -> bool:
        """
        检查两行是否相似（可能是重复行）
        增强版：更宽松的相似度判断
        """
        # 如果长度差异太大，肯定不相似
        if abs(len(row1) - len(row2)) > 2:  # 允许更大的列数差异
            return False

        # 计算相似单元格的比例
        min_len = min(len(row1), len(row2))
        similar_cells = 0

        for i in range(min_len):
            # 更宽松的文本相似度检查
            text1 = row1[i].strip().lower()
            text2 = row2[i].strip().lower()

            # 如果文本长度相似且有足够多的共同词
            if (0.5 < len(text1)/max(len(text2), 1) < 2 and  # 长度比例在0.5-2之间
                self._texts_have_common_words(text1, text2)):
                similar_cells += 1

        # 如果60%以上的单元格相似，则认为行相似（降低阈值提高检测率）
        return similar_cells / min_len >= 0.6

    def _texts_have_common_words(self, text1: str, text2: str) -> bool:
        """
        检查两个文本是否有共同单词
        """
        # 提取单词
        words1 = set(re.findall(r'\w+', text1))
        words2 = set(re.findall(r'\w+', text2))

        # 如果有共同单词或者文本内容非常接近
        if words1 & words2:
            return True

        # 对于较短的文本，直接比较相似度
        if len(text1) < 20 and len(text2) < 20:
            # 计算编辑距离（简化版）
            if abs(len(text1) - len(text2)) <= 2 and self._levenshtein_distance(text1, text2) <= 3:
                return True

        return False

    def _levenshtein_distance(self, s1: str, s2: str) -> int:
        """
        计算两个字符串的编辑距离（简化版）
        """
        if len(s1) < len(s2):
            return self._levenshtein_distance(s2, s1)

        if len(s2) == 0:
            return len(s1)

        previous_row = range(len(s2) + 1)
        for i, c1 in enumerate(s1):
            current_row = [i + 1]
            for j, c2 in enumerate(s2):
                insertions = previous_row[j + 1] + 1
                deletions = current_row[j] + 1
                substitutions = previous_row[j] + (c1 != c2)
                current_row.append(min(insertions, deletions, substitutions))
            previous_row = current_row

        return previous_row[-1]

    def save_merged_tables_to_csv(self, merged_tables, output_dir: str = "."):
        """
        将合并后的表格保存为CSV文件
        """
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)

        for (page1, page2), table_data in merged_tables.items():
            if table_data:
                csv_filename = os.path.join(output_dir, f"enhanced_merged_table_pages_{page1+1}_{page2+1}.csv")

                with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile:
                    writer = csv.writer(csvfile)
                    for row in table_data:
                        writer.writerow(row)

                print(f"已将合并后的表格保存到: {csv_filename}")

    def debug_cross_page_detection(self, page1_num: int, page2_num: int, margin_threshold: float = 30.0):
        """
        调试跨页表格检测
        """
        print(f"\n=== 调试跨页表格检测：第 {page1_num+1} 页和第 {page2_num+1} 页 ===")

        # 提取页面元素
        page1_elements = self.extract_page_elements(page1_num)
        page2_elements = self.extract_page_elements(page2_num)

        # 显示页面基本信息
        print(f"页面 {page1_num+1} 文本块数量: {len(page1_elements['text_blocks'])}")
        print(f"页面 {page2_num+1} 文本块数量: {len(page2_elements['text_blocks'])}")
        print(f"页面 {page1_num+1} 线条数量: {len(page1_elements['lines'])}")
        print(f"页面 {page2_num+1} 线条数量: {len(page2_elements['lines'])}")

        # 显示页面底部和顶部的文本块内容
        page1_rect = self.pages[page1_num].rect
        page2_rect = self.pages[page2_num].rect

        # 页面1底部文本块
        bottom_texts = []
        for block in page1_elements["text_blocks"]:
            if block["bbox"][3] > page1_rect.y1 - margin_threshold * 3:
                bottom_texts.append(block)

        # 页面2顶部文本块
        top_texts = []
        for block in page2_elements["text_blocks"]:
            if block["bbox"][1] < page2_rect.y0 + margin_threshold * 3:
                top_texts.append(block)

        print(f"\n页面 {page1_num+1} 底部文本块内容:")
        for i, block in enumerate(bottom_texts):
            print(f"文本块 {i+1}: {block['text'][:100]}...")
            print(f"  位置: {block['bbox']}")

        print(f"\n页面 {page2_num+1} 顶部文本块内容:")
        for i, block in enumerate(top_texts):
            print(f"文本块 {i+1}: {block['text'][:100]}...")
            print(f"  位置: {block['bbox']}")

        # 分析文本连续性
        text_continuity = self._check_cross_page_text_structure(page1_num, page2_num, margin_threshold)
        print(f"\n文本结构连续性检测结果: {text_continuity}")

        print("\n=== 调试结束 ===")

# 测试脚本
if __name__ == "__main__":
    import sys

    if len(sys.argv) < 2:
        print("用法: python enhanced_table_detector.py <pdf文件路径> [输出目录] [margin_threshold]")
        print("示例: python enhanced_table_detector.py 'D:\\工作\\02-文档智能\\05-测试数据\\01-跨页表格\\跨页表格01.pdf'")
        sys.exit(1)

    pdf_path = sys.argv[1]
    output_dir = sys.argv[2] if len(sys.argv) > 2 else os.path.dirname(pdf_path)
    margin_threshold = float(sys.argv[3]) if len(sys.argv) > 3 else 30.0

    if not os.path.exists(pdf_path):
        print(f"错误: 找不到PDF文件 '{pdf_path}'")
        sys.exit(1)

    # 确保输出目录存在
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    try:
        print(f"开始处理PDF文件: {pdf_path}")
        print(f"使用参数: margin_threshold={margin_threshold}")
        detector = EnhancedTableDetector(pdf_path)

        # 首先进行调试分析
        if len(detector.pages) > 1:
            # 调试第一页和第二页之间的跨页表格检测
            detector.debug_cross_page_detection(0, 1, margin_threshold)

        # 检测并合并跨页表格
        print("\n正在检测跨页表格...")
        merged_tables = detector.merge_cross_page_tables(margin_threshold)

        if merged_tables:
            print(f"\n成功检测并合并了 {len(merged_tables)} 个跨页表格")

            # 保存结果
            detector.save_merged_tables_to_csv(merged_tables, output_dir)
        else:
            print("\n未检测到跨页表格。尝试以下方法:\n" \
                 "1. 调整margin_threshold参数（当前值: {margin_threshold}）\n" \
                 "2. 检查PDF文件是否包含表格\n" \
                 "3. 尝试使用不同的PDF阅读器确认表格是否存在")

            # 提供更多调试选项
            print("\n调试建议:\n")
            print("1. 尝试使用更大的margin_threshold值，例如:")
            print(f"   python enhanced_table_detector.py '{pdf_path}' '{output_dir}' 40")
            print("2. 检查PDF文件的结构，某些加密或特殊格式的PDF可能需要特殊处理")

    except Exception as e:
        print(f"处理PDF时发生错误: {e}")
        import traceback
        traceback.print_exc()